<!DOCTYPE html>
<html lang="en">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 5.4.0">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/logo.svg" color="#222">
  <meta name="google-site-verification" content="d44tDfSSWxm1_XP1dAq65hkgyD6zw70Ua9JdCaJqWGg">

<link rel="stylesheet" href="/css/main.css">


<link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">

<script id="hexo-configurations">
    var NexT = window.NexT || {};
    var CONFIG = {"hostname":"zasdfgbnm.github.io","root":"/","scheme":"Muse","version":"7.8.0","exturl":false,"sidebar":{"position":"left","display":"post","padding":18,"offset":12,"onmobile":false},"copycode":{"enable":false,"show_result":false,"style":null},"back2top":{"enable":true,"sidebar":false,"scrollpercent":false},"bookmark":{"enable":false,"color":"#222","save":"auto"},"fancybox":false,"mediumzoom":false,"lazyload":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":false,"nav":null},"algolia":{"hits":{"per_page":10},"labels":{"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}},"localsearch":{"enable":false,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false},"motion":{"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}}};
  </script>

  <meta name="description" content="这篇是阅读PyTorch源代码整理的笔记，方便以后翻阅。这里主要是想知道PyTorch的operators的定义都是怎么组织的，以及如果要添加新的operator的话，该怎么做。">
<meta property="og:type" content="article">
<meta property="og:title" content="从头开始阅读PyTorch代码 -- Operators篇">
<meta property="og:url" content="https://zasdfgbnm.github.io/2018/06/11/%E4%BB%8E%E5%A4%B4%E5%BC%80%E5%A7%8B%E9%98%85%E8%AF%BBPyTorch%E4%BB%A3%E7%A0%81%20--%20Operators%E7%AF%87/index.html">
<meta property="og:site_name" content="zasdfgbnm">
<meta property="og:description" content="这篇是阅读PyTorch源代码整理的笔记，方便以后翻阅。这里主要是想知道PyTorch的operators的定义都是怎么组织的，以及如果要添加新的operator的话，该怎么做。">
<meta property="og:locale" content="en_US">
<meta property="article:published_time" content="2018-06-11T12:50:00.000Z">
<meta property="article:modified_time" content="2021-04-04T05:17:59.766Z">
<meta property="article:author" content="zasdfgbnm">
<meta property="article:tag" content="机器学习">
<meta property="article:tag" content="PyTorch">
<meta property="article:tag" content="深度学习">
<meta name="twitter:card" content="summary">

<link rel="canonical" href="https://zasdfgbnm.github.io/2018/06/11/%E4%BB%8E%E5%A4%B4%E5%BC%80%E5%A7%8B%E9%98%85%E8%AF%BBPyTorch%E4%BB%A3%E7%A0%81%20--%20Operators%E7%AF%87/">


<script id="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome : false,
    isPost : true,
    lang   : 'en'
  };
</script>

  <title>从头开始阅读PyTorch代码 -- Operators篇 | zasdfgbnm</title>
  
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-7583294-5"></script>
    <script>
      if (CONFIG.hostname === location.hostname) {
        window.dataLayer = window.dataLayer || [];
        function gtag(){dataLayer.push(arguments);}
        gtag('js', new Date());
        gtag('config', 'UA-7583294-5');
      }
    </script>


  <script>
    var _hmt = _hmt || [];
    (function() {
      var hm = document.createElement("script");
      hm.src = "https://hm.baidu.com/hm.js?a56abdeb557a286a6b7a104348fdfbcd";
      var s = document.getElementsByTagName("script")[0];
      s.parentNode.insertBefore(hm, s);
    })();
  </script>




  <noscript>
  <style>
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header { opacity: initial; }

  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>

<link rel="alternate" href="/atom.xml" title="zasdfgbnm" type="application/atom+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container use-motion">
    <div class="headband"></div>

    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-nav-toggle">
    <div class="toggle" aria-label="Toggle navigation bar">
      <span class="toggle-line toggle-line-first"></span>
      <span class="toggle-line toggle-line-middle"></span>
      <span class="toggle-line toggle-line-last"></span>
    </div>
  </div>

  <div class="site-meta">

    <a href="/" class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <h1 class="site-title">zasdfgbnm</h1>
      <span class="logo-line-after"><i></i></span>
    </a>
  </div>

  <div class="site-nav-right">
    <div class="toggle popup-trigger">
    </div>
  </div>
</div>




<nav class="site-nav">
  <ul id="menu" class="main-menu menu">
        <li class="menu-item menu-item-home">

    <a href="/" rel="section"><i class="fa fa-home fa-fw"></i>Home</a>

  </li>
        <li class="menu-item menu-item-archives">

    <a href="/archives/" rel="section"><i class="fa fa-archive fa-fw"></i>Archives</a>

  </li>
        <li class="menu-item menu-item-about">

    <a href="/about/" rel="section"><i class="fa fa-user fa-fw"></i>About</a>

  </li>
        <li class="menu-item menu-item-tags">

    <a href="/tags/" rel="section"><i class="fa fa-tags fa-fw"></i>Tags</a>

  </li>
        <li class="menu-item menu-item-categories">

    <a href="/categories/" rel="section"><i class="fa fa-th fa-fw"></i>Categories</a>

  </li>
        <li class="menu-item menu-item-sitemap">

    <a href="/sitemap.xml" rel="section"><i class="fa fa-sitemap fa-fw"></i>Sitemap</a>

  </li>
  </ul>
</nav>




</div>
    </header>

    
  <div class="back-to-top">
    <i class="fa fa-arrow-up"></i>
    <span>0%</span>
  </div>


    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          

          <div class="content post posts-expand">
            

    
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="en">
    <link itemprop="mainEntityOfPage" href="https://zasdfgbnm.github.io/2018/06/11/%E4%BB%8E%E5%A4%B4%E5%BC%80%E5%A7%8B%E9%98%85%E8%AF%BBPyTorch%E4%BB%A3%E7%A0%81%20--%20Operators%E7%AF%87/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="zasdfgbnm">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="zasdfgbnm">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          从头开始阅读PyTorch代码 -- Operators篇
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-calendar"></i>
              </span>
              <span class="post-meta-item-text">Posted on</span>

              <time title="Created: 2018-06-11 08:50:00" itemprop="dateCreated datePublished" datetime="2018-06-11T08:50:00-04:00">2018-06-11</time>
            </span>
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="far fa-calendar-check"></i>
                </span>
                <span class="post-meta-item-text">Edited on</span>
                <time title="Modified: 2021-04-04 01:17:59" itemprop="dateModified" datetime="2021-04-04T01:17:59-04:00">2021-04-04</time>
              </span>
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="far fa-folder"></i>
              </span>
              <span class="post-meta-item-text">In</span>
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0/" itemprop="url" rel="index"><span itemprop="name">机器学习</span></a>
                </span>
            </span>

          
  
  <span class="post-meta-item">
    
      <span class="post-meta-item-icon">
        <i class="far fa-comment"></i>
      </span>
      <span class="post-meta-item-text">Disqus: </span>
    
    <a title="disqus" href="/2018/06/11/%E4%BB%8E%E5%A4%B4%E5%BC%80%E5%A7%8B%E9%98%85%E8%AF%BBPyTorch%E4%BB%A3%E7%A0%81%20--%20Operators%E7%AF%87/#disqus_thread" itemprop="discussionUrl">
      <span class="post-comments-count disqus-comment-count" data-disqus-identifier="2018/06/11/从头开始阅读PyTorch代码 -- Operators篇/" itemprop="commentCount"></span>
    </a>
  </span>
  
  

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
        <p>这篇是阅读PyTorch源代码整理的笔记，方便以后翻阅。这里主要是想知道PyTorch的operators的定义都是怎么组织的，以及如果要添加新的operator的话，该怎么做。</p>
<span id="more"></span>

<p>由于pytorch开发非常活跃，代码也在不断地更改，本文内容跟读者实际看到的最新的代码肯定也有所区别。如果读者想要查看作者写文时候的代码，可以在pytorch仓库中:</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">git checkout 14cbd9adb8efafbd51444fdd88d6a34e6438b1c5</span><br></pre></td></tr></table></figure>

<h1 id="init-py跟setup-py"><a href="#init-py跟setup-py" class="headerlink" title="__init__.py跟setup.py"></a><code>__init__.py</code>跟<code>setup.py</code></h1><p>比较不错的着手点是<code>torch</code>这个模块的<code>__init__.py</code>跟安装用的<code>setup.py</code>。 把两个文件都浏览一遍有个大体的概念。然后在<code>__init__.py</code>里面搜<code>__all__</code>，通过观察这些operators是怎么被添加到<code>__all__</code>里面去的，就能知道我们用的所有的那些个operators是怎么来的了。</p>
<p>搜<code>__all__</code>发现的关键的一段是：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> torch._C <span class="keyword">import</span> *</span><br><span class="line"></span><br><span class="line">__all__ += [name <span class="keyword">for</span> name <span class="keyword">in</span> <span class="built_in">dir</span>(_C)</span><br><span class="line">            <span class="keyword">if</span> name[<span class="number">0</span>] != <span class="string">&#x27;_&#x27;</span> <span class="keyword">and</span></span><br><span class="line">            <span class="keyword">not</span> name.endswith(<span class="string">&#x27;Base&#x27;</span>)]</span><br></pre></td></tr></table></figure>
<p>这段干的事情就是把<code>torch._C</code>中定义的各种东西按需加入<code>__all__</code>里面去，所以要想知道哪些operators是怎么来的，还是需要去看<code>torch._C</code>。从名字一看就知道，<code>torch._C</code>这个东西，是PyTorch的用C/C++之类的语言写的那一部分。这就涉及到这一部分是怎么构建的了，这个要从<code>setup.py</code>里面翻找，发现的相关的代码段如下：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line">main_sources = [</span><br><span class="line">    <span class="string">&quot;torch/csrc/PtrWrapper.cpp&quot;</span>,</span><br><span class="line">    <span class="comment">#......</span></span><br><span class="line">]</span><br><span class="line"></span><br><span class="line"><span class="comment">#.....</span></span><br><span class="line"></span><br><span class="line">extensions = []</span><br><span class="line">packages = find_packages(exclude=(<span class="string">&#x27;tools&#x27;</span>, <span class="string">&#x27;tools.*&#x27;</span>, <span class="string">&#x27;caffe2&#x27;</span>, <span class="string">&#x27;caffe2.*&#x27;</span>, <span class="string">&#x27;caffe&#x27;</span>, <span class="string">&#x27;caffe.*&#x27;</span>))</span><br><span class="line">C = Extension(<span class="string">&quot;torch._C&quot;</span>,</span><br><span class="line">              libraries=main_libraries,</span><br><span class="line">              sources=main_sources,</span><br><span class="line">              language=<span class="string">&#x27;c++&#x27;</span>,</span><br><span class="line">              extra_compile_args=main_compile_args + extra_compile_args,</span><br><span class="line">              include_dirs=include_dirs,</span><br><span class="line">              library_dirs=library_dirs,</span><br><span class="line">              extra_link_args=extra_link_args + main_link_args + [make_relative_rpath(<span class="string">&#x27;lib&#x27;</span>)],</span><br><span class="line">              )</span><br><span class="line">extensions.append(C)</span><br></pre></td></tr></table></figure>
<p>基本上就可以断定<code>torch._C</code>这个东西就是从<code>torch/csrc/</code>这个目录里面的一大堆文件编译出来的了。<code>torch/csrc/</code>里面文件一大堆，从哪里着手是个问题。因为<code>torch._C</code>是python的一个模块，那么肯定得有地方通过python的C-binding创建这个模块才是，这就是个不错的着手点。要找到这个模块是从哪里创建的，这时候就要祭出grep大法了，在<code>torch/csrc/</code>这个目录里面运行这样一条命令：</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">grep <span class="string">&#x27;torch._C&#x27;</span> -r .</span><br></pre></td></tr></table></figure>
<p>就可以得到所有有关的代码行了，发现的最像的一行是：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">.&#x2F;Module.cpp:  ASSERT_TRUE(module &#x3D; Py_InitModule(&quot;torch._C&quot;, methods.data()));</span><br></pre></td></tr></table></figure>
<p>这一行一看就是在初始化这个模块，而且文件名叫做<code>Module.cpp</code>也很符合，那下一步就从这里开始好了。<code>__init__.py</code>跟<code>setup.py</code>也可以退出我们的历史舞台了。</p>
<p>另外要注意，在施展grep大法之前，一定要先把PyTorch给编译一遍，因为PyTorch的很多代码是编译的时候根据其他文件生成的，带着生成的文件一起查找比较好。</p>
<h1 id="Module-cpp跟autograd"><a href="#Module-cpp跟autograd" class="headerlink" title="Module.cpp跟autograd"></a><code>Module.cpp</code>跟autograd</h1><p>把这个文件从头到尾浏览一遍，基本上就可以断定初始化模块是在<code>initModule</code>里面完成的了，这个函数里面初始化了一大堆东西，主要还是找找具体的哪一行是负责初始化那堆operators的。注意到<code>initModule</code>里面有好多类似</p>
<figure class="highlight c++"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#<span class="meta-keyword">ifdef</span> WITH_CUDA</span></span><br><span class="line">  <span class="comment">// do something ....</span></span><br><span class="line"><span class="meta">#<span class="meta-keyword">endif</span></span></span><br></pre></td></tr></table></figure>
<p>这种，这种就可以直接跳过不读了，因为不管你是否启用了这一堆的feature，那些个operators都是存在的，那就当你没启用这堆好了。还有一堆东西，看名字就不相关，就直接不用搭理了。所以到最后基本上筛选出来的看起来可能是的也只有：</p>
<figure class="highlight c++"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="built_in">THPUtils_addPyMethodDefs</span>(methods, TorchMethods);</span><br><span class="line"><span class="built_in">THPUtils_addPyMethodDefs</span>(methods, torch::autograd::<span class="built_in">python_functions</span>());</span><br><span class="line"><span class="comment">//....</span></span><br><span class="line"><span class="built_in">ASSERT_TRUE</span>(<span class="built_in">THPVariable_initModule</span>(<span class="keyword">module</span>));</span><br><span class="line"><span class="built_in">ASSERT_TRUE</span>(<span class="built_in">THPFunction_initModule</span>(<span class="keyword">module</span>));</span><br></pre></td></tr></table></figure>
<p>那就先从<code>TorchMethods</code>看起，这个东西就定义在<code>Module.cpp</code>里面，看一眼就会知道跟operators没啥关系。<code>torch::autograd::python_functions()</code>这个东西，使用grep大法，可以发现他的定义位于<code>torch/csrc/autograd/init.cpp</code>，也不是啥想要的，继续看<code>THPVariable_initModule</code>，使用grep大法，就会发现这个函数是在<code>torch/csrc/autograd/python_variable.cpp</code>里面定义的，翻看一下这个函数的定义，注意到下面这行：</p>
<figure class="highlight c++"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="built_in">THPUtils_addPyMethodDefs</span>(methods, torch::autograd::variable_methods);</span><br></pre></td></tr></table></figure>
<p>整个函数定义里面，也只有这一行最像是定义operators的了，于是继续深挖，继续使用grep大法：</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">grep variable_methods -r .</span><br></pre></td></tr></table></figure>
<p>就会发现这个变量是定义在<code>torch/csrc/</code>下的<code>autograd/generated/python_variable_methods.cpp</code>里面的，打开看看，发现如下的内容：</p>
<figure class="highlight c++"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">PyMethodDef variable_methods[] = &#123;</span><br><span class="line">  &#123;<span class="string">&quot;__add__&quot;</span>, (PyCFunction)THPVariable_add, METH_VARARGS | METH_KEYWORDS, <span class="literal">NULL</span>&#125;,</span><br><span class="line">  <span class="comment">//......</span></span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<p>这就是一个长长的列表，列举了所有的的operators，每个operator对应一个<code>THPVariable_</code>开头的函数定义在同一个文件里面，而这个文件的开头，则说明了这个文件是从<code>tools/autograd/templates/python_variable_methods.cpp</code>这个模板生成而来。</p>
<p>浏览一下所有的<code>THPVariable_</code>开头的函数，就会发现所有的不同的这些个函数都大同小异，基本上核心部分只有下面的内容：</p>
<figure class="highlight c++"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="built_in">wrap</span>(<span class="built_in">dispatch_xxxxx</span>(...));</span><br></pre></td></tr></table></figure>

<p>其中<code>dispatch_xxxxx</code>应该就是<code>xxxxx</code>这个operator的核心实现部分。继续动用grep大法挖，只需要随便挑选一个operator搜索就行了，例如：</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">grep dispatch_acos -r .</span><br></pre></td></tr></table></figure>
<p>搜了就会发现这些个<code>dispatch_</code>开头的函数，是定义在同目录以下的<code>python_variable_methods_dispatch.h</code>文件里面的。翻开这个文件，浏览一下这些dispatch函数的定义，都大同小异，下面摘录其中一个：</p>
<figure class="highlight c++"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">inline</span> Tensor <span class="title">dispatch_add</span><span class="params">(Tensor &amp; self, Scalar alpha, <span class="keyword">const</span> Tensor &amp; other)</span> </span>&#123;</span><br><span class="line"></span><br><span class="line">  AutoNoGIL no_gil;</span><br><span class="line">  <span class="function">AutoGPU <span class="title">auto_gpu</span><span class="params">(self)</span></span>;</span><br><span class="line">  <span class="keyword">return</span> self.<span class="built_in">add</span>(other, alpha);</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<p>从代码发现，这些个operator，实际上是<code>Tensor</code>这个类的成员函数，所以我们就知道，下一步应该挖的，就是<code>Tensor</code>这个类了。除此以外，还有一个很重要的东西就是搞明白代码生成的原理，这样就能知道代码生成器是怎样找到这些operators的定义，进而生成这些函数的了。</p>
<p>Tensor这个类的出处在<code>python_variable_methods_dispatch.h</code>文件的头部可以找到：</p>
<figure class="highlight c++"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">using</span> at::Tensor;</span><br></pre></td></tr></table></figure>
<p>由此可见Tensor是ATen里面定义的，由此看来autograd也基本要退出我们的历史舞台了，轮到ATen登场了。</p>
<h1 id="ATen"><a href="#ATen" class="headerlink" title="ATen"></a>ATen</h1><p>要学习ATen其实非常简单，在<code>aten</code>目录里面乱扒乱翻一通，挨个文件夹都点开瞅两眼，把所有的<code>README.md</code>都读一遍，就会发现，实际上ATen的算符是怎么定义的，实际上，已经在<code>aten/src/ATen/native/README.md</code>文件中，进行了非常详细的说明。</p>
<p>综合各个<code>README.md</code>的信息，并简单总结一下，就是：PyTorch的算符，都是定义在ATen里面的，而ATen里面的算符的实现，一部分是从老的Lua Torch继承而来，这一部分的代码，位于<code>aten/src/TH*</code>这些个目录里面，这些都是历史遗留的遗产，继承过来直接用，并不是PyTorch最终想要的operator的实现方式。最终“好”的实现方式，是在<code>aten/src/ATen/native/</code>目录里面。很多算符，也已经在这个目录下，被重新实现了一遍。这些老的算符的列表，是在<code>aten/src/ATen/Declarations.cwrap</code>中定义的。而新的算符的列表的定义，是在<code>aten/src/ATen/native/native_functions.yaml</code>中。本文只去探讨新的算符的实现方式。</p>
<p>到了这里，想要继续扒一下新的算符实现的，就需要去扒一下<code>native_functions.yaml</code>这个文件是怎么被读取的了。在PyTorch的根目录下，继续用grep大法，搜索关键字<code>native_functions.yaml</code>，得到的结果中，一条看起来很像是我们想要的结果的是：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">.&#x2F;aten&#x2F;src&#x2F;ATen&#x2F;gen.py:    native_files &#x3D; filter_by_extension(options.files, &#39;native_functions.yaml&#39;)</span><br></pre></td></tr></table></figure>

<p>打开<code>gen.py</code>这个文件查看，就会发现下面比较有趣的代码：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">TEMPLATE_PATH = options.source_path + <span class="string">&quot;/templates&quot;</span></span><br><span class="line"><span class="comment"># ......</span></span><br><span class="line">TENSOR_DERIVED_H = CodeTemplate.from_file(TEMPLATE_PATH + <span class="string">&quot;/TensorDerived.h&quot;</span>)</span><br><span class="line">TENSOR_H = CodeTemplate.from_file(TEMPLATE_PATH + <span class="string">&quot;/Tensor.h&quot;</span>)</span><br><span class="line">TENSOR_METHODS_H = CodeTemplate.from_file(TEMPLATE_PATH + <span class="string">&quot;/TensorMethods.h&quot;</span>)</span><br></pre></td></tr></table></figure>
<p>以及：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">generate_outputs</span>():</span></span><br><span class="line">    cwrap_files = filter_by_extension(options.files, <span class="string">&#x27;.cwrap&#x27;</span>)</span><br><span class="line">    nn_files = filter_by_extension(options.files, <span class="string">&#x27;nn.yaml&#x27;</span>, <span class="string">&#x27;.h&#x27;</span>)</span><br><span class="line">    native_files = filter_by_extension(options.files, <span class="string">&#x27;native_functions.yaml&#x27;</span>)</span><br><span class="line"></span><br><span class="line">    declarations = [d</span><br><span class="line">                    <span class="keyword">for</span> file <span class="keyword">in</span> cwrap_files</span><br><span class="line">                    <span class="keyword">for</span> d <span class="keyword">in</span> cwrap_parser.parse(file)]</span><br><span class="line"></span><br><span class="line">    declarations += nn_parse.run(nn_files)</span><br><span class="line">    declarations += native_parse.run(native_files)</span><br><span class="line">    declarations = preprocess_declarations.run(declarations)</span><br><span class="line">    <span class="comment"># ......</span></span><br></pre></td></tr></table></figure>

<p>继续在这附近翻找上下文，就会发现，ATen的代码生成，是通过<code>gen.py</code>等的Python脚本，解析之前说过的那若干个列表文件，然后根据<code>aten/src/ATen/templates/</code>目录下的文件生成的。这些文件，都是模板，不长，全都浏览一遍就是了。阅读的过程，就会发现非常多的重要信息，比如在<code>Tensor.h</code>中有：</p>
<figure class="highlight c++"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">namespace</span> at &#123;</span><br><span class="line"><span class="comment">// ......</span></span><br><span class="line"><span class="class"><span class="keyword">struct</span> <span class="title">Tensor</span> :</span> <span class="keyword">public</span> detail::TensorBase &#123;</span><br><span class="line"><span class="comment">// ......</span></span><br><span class="line">$&#123;tensor_method_declarations&#125;</span><br><span class="line"><span class="comment">// ......</span></span><br><span class="line">&#125;;</span><br><span class="line"><span class="comment">// ......</span></span><br><span class="line">&#125; <span class="comment">// namespace at</span></span><br></pre></td></tr></table></figure>

<p>以及<code>TensorMethods.h</code>中的：</p>
<figure class="highlight c++"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">namespace</span> at &#123;</span><br><span class="line"></span><br><span class="line"><span class="keyword">inline</span> Tensor &amp; Tensor::<span class="keyword">operator</span>=(Tensor <span class="keyword">const</span> &amp; rhs) &amp;&amp; &#123;</span><br><span class="line">  <span class="keyword">return</span> <span class="built_in">copy_</span>(rhs);</span><br><span class="line">&#125;</span><br><span class="line"><span class="comment">// ......</span></span><br><span class="line"><span class="comment">// all static inline to allow for inlining of the non-dynamic part of dispatch</span></span><br><span class="line">$&#123;tensor_method_definitions&#125;</span><br><span class="line"><span class="comment">// ......</span></span><br><span class="line">&#125; <span class="comment">// namespace at</span></span><br></pre></td></tr></table></figure>

<p>基本上，上面的看完，就已经知道，Tensor类是怎么定义的了。最后一步，就是看一下<code>tensor_method_definitions</code>是怎么填充的了。</p>
<p>继续动用grep大法扒，在PyTorch的根目录用grep搜索<code>tensor_method_definitions</code>，会得到下面有意思的结果：</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">.&#x2F;aten&#x2F;src&#x2F;ATen&#x2F;function_wrapper.py:            top_env[&#39;tensor_method_definitions&#39;].append(</span><br></pre></td></tr></table></figure>

<p>看了这个，直接跳到<code>function_wrapper.py</code>文件去扒，发现下面一段：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">if</span> is_method:</span><br><span class="line">    top_env[<span class="string">&#x27;tensor_method_declarations&#x27;</span>].append(</span><br><span class="line">        TENSOR_METHOD_DECLARATION.substitute(env))</span><br><span class="line">    top_env[<span class="string">&#x27;tensor_method_definitions&#x27;</span>].append(</span><br><span class="line">        TENSOR_METHOD_DEFINITION.substitute(env))</span><br><span class="line">    method_of.append(<span class="string">&#x27;Tensor&#x27;</span>)</span><br></pre></td></tr></table></figure>

<p>而上面代码中的<code>TENSOR_METHOD_DECLARATION</code>跟<code>TENSOR_METHOD_DEFINITION</code>，就定义在同一个文件中：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># add non-virtual declaration to Tensor.h</span></span><br><span class="line">TENSOR_METHOD_DECLARATION = CodeTemplate(<span class="string">&quot;&quot;&quot;\</span></span><br><span class="line"><span class="string">$&#123;return_type&#125; $&#123;api_name&#125;($&#123;method_formals_with_defaults&#125;)$&#123;const_mark&#125;;</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span>)</span><br><span class="line"><span class="comment"># add non-virtual declaration to Tensor.cpp</span></span><br><span class="line">TENSOR_METHOD_DEFINITION = CodeTemplate(<span class="string">&quot;&quot;&quot;\</span></span><br><span class="line"><span class="string">inline $&#123;return_type&#125; Tensor::$&#123;api_name&#125;($&#123;method_formals&#125;)$&#123;const_mark&#125; &#123;</span></span><br><span class="line"><span class="string">    return type().$&#123;api_name&#125;($&#123;method_actuals&#125;);</span></span><br><span class="line"><span class="string">&#125;</span></span><br><span class="line"><span class="string">&quot;&quot;&quot;</span>)</span><br></pre></td></tr></table></figure>

<p>至此，基本上，整个ATen的代码生成，基本上算是扒完了，具体做事情的时候，再去返回这些涉及到的文件查看即可。</p>
<p>本文完结</p>

    </div>

    
    
    
        

  <div class="followme">
    <p>Welcome to my other publishing channels</p>

    <div class="social-list">

        <div class="social-item">
          <a target="_blank" class="social-link" href="https://twitter.com/gaoxiang_ai">
            <span class="icon">
              <i class="fab fa-twitter"></i>
            </span>

            <span class="label">Twitter</span>
          </a>
        </div>

        <div class="social-item">
          <a target="_blank" class="social-link" href="/atom.xml">
            <span class="icon">
              <i class="fa fa-rss"></i>
            </span>

            <span class="label">RSS</span>
          </a>
        </div>
    </div>
  </div>


      <footer class="post-footer">
          <div class="post-tags">
              <a href="/tags/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0/" rel="tag"># 机器学习</a>
              <a href="/tags/PyTorch/" rel="tag"># PyTorch</a>
              <a href="/tags/%E6%B7%B1%E5%BA%A6%E5%AD%A6%E4%B9%A0/" rel="tag"># 深度学习</a>
          </div>

        


        
    <div class="post-nav">
      <div class="post-nav-item">
    <a href="/2017/12/29/Using-docker-image-as-your-desktop-system/" rel="prev" title="Using docker image as your desktop system">
      <i class="fa fa-chevron-left"></i> Using docker image as your desktop system
    </a></div>
      <div class="post-nav-item">
    <a href="/2018/09/20/PyTorch-JIT-Source-Code-Read-Note/" rel="next" title="PyTorch JIT Source Code Read Note">
      PyTorch JIT Source Code Read Note <i class="fa fa-chevron-right"></i>
    </a></div>
    </div>
      </footer>
    
  </article>
  
  
  



          </div>
          
    
  <div class="comments">
    <div id="disqus_thread">
      <noscript>Please enable JavaScript to view the comments powered by Disqus.</noscript>
    </div>
  </div>
  

<script>
  window.addEventListener('tabs:register', () => {
    let { activeClass } = CONFIG.comments;
    if (CONFIG.comments.storage) {
      activeClass = localStorage.getItem('comments_active') || activeClass;
    }
    if (activeClass) {
      let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
      if (activeTab) {
        activeTab.click();
      }
    }
  });
  if (CONFIG.comments.storage) {
    window.addEventListener('tabs:click', event => {
      if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
      let commentClass = event.target.classList[1];
      localStorage.setItem('comments_active', commentClass);
    });
  }
</script>

        </div>
          
  
  <div class="toggle sidebar-toggle">
    <span class="toggle-line toggle-line-first"></span>
    <span class="toggle-line toggle-line-middle"></span>
    <span class="toggle-line toggle-line-last"></span>
  </div>

  <aside class="sidebar">
    <div class="sidebar-inner">

      <ul class="sidebar-nav motion-element">
        <li class="sidebar-nav-toc">
          Table of Contents
        </li>
        <li class="sidebar-nav-overview">
          Overview
        </li>
      </ul>

      <!--noindex-->
      <div class="post-toc-wrap sidebar-panel">
          <div class="post-toc motion-element"><ol class="nav"><li class="nav-item nav-level-1"><a class="nav-link" href="#init-py%E8%B7%9Fsetup-py"><span class="nav-number">1.</span> <span class="nav-text">__init__.py跟setup.py</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#Module-cpp%E8%B7%9Fautograd"><span class="nav-number">2.</span> <span class="nav-text">Module.cpp跟autograd</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#ATen"><span class="nav-number">3.</span> <span class="nav-text">ATen</span></a></li></ol></div>
      </div>
      <!--/noindex-->

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
  <p class="site-author-name" itemprop="name">zasdfgbnm</p>
  <div class="site-description" itemprop="description"></div>
</div>
<div class="site-state-wrap motion-element">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
          <a href="/archives/">
        
          <span class="site-state-item-count">11</span>
          <span class="site-state-item-name">posts</span>
        </a>
      </div>
      <div class="site-state-item site-state-categories">
            <a href="/categories/">
          
        <span class="site-state-item-count">5</span>
        <span class="site-state-item-name">categories</span></a>
      </div>
      <div class="site-state-item site-state-tags">
            <a href="/tags/">
          
        <span class="site-state-item-count">56</span>
        <span class="site-state-item-name">tags</span></a>
      </div>
  </nav>
</div>
  <div class="links-of-author motion-element">
      <span class="links-of-author-item">
        <a href="https://github.com/zasdfgbnm" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;zasdfgbnm" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>GitHub</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://twitter.com/gaoxiang_ai" title="Twitter → https:&#x2F;&#x2F;twitter.com&#x2F;gaoxiang_ai" rel="noopener" target="_blank"><i class="fab fa-twitter fa-fw"></i>Twitter</a>
      </span>
  </div>



      </div>

    </div>
  </aside>
  <div id="sidebar-dimmer"></div>


      </div>
    </main>

    <footer class="footer">
      <div class="footer-inner">
        

        

<div class="copyright">
  
  &copy; 
  <span itemprop="copyrightYear">2021</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">zasdfgbnm</span>
</div>
  <div class="powered-by">Powered by <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://muse.theme-next.org/" class="theme-link" rel="noopener" target="_blank">NexT.Muse</a>
  </div>

        








      </div>
    </footer>
  </div>

  
  <script src="/lib/anime.min.js"></script>
  <script src="/lib/velocity/velocity.min.js"></script>
  <script src="/lib/velocity/velocity.ui.min.js"></script>

<script src="/js/utils.js"></script>

<script src="/js/motion.js"></script>


<script src="/js/schemes/muse.js"></script>


<script src="/js/next-boot.js"></script>




  
  <script>
    (function(){
      var canonicalURL, curProtocol;
      //Get the <link> tag
      var x=document.getElementsByTagName("link");
		//Find the last canonical URL
		if(x.length > 0){
			for (i=0;i<x.length;i++){
				if(x[i].rel.toLowerCase() == 'canonical' && x[i].href){
					canonicalURL=x[i].href;
				}
			}
		}
    //Get protocol
	    if (!canonicalURL){
	    	curProtocol = window.location.protocol.split(':')[0];
	    }
	    else{
	    	curProtocol = canonicalURL.split(':')[0];
	    }
      //Get current URL if the canonical URL does not exist
	    if (!canonicalURL) canonicalURL = window.location.href;
	    //Assign script content. Replace current URL with the canonical URL
      !function(){var e=/([http|https]:\/\/[a-zA-Z0-9\_\.]+\.baidu\.com)/gi,r=canonicalURL,t=document.referrer;if(!e.test(r)){var n=(String(curProtocol).toLowerCase() === 'https')?"https://sp0.baidu.com/9_Q4simg2RQJ8t7jm9iCKT-xh_/s.gif":"//api.share.baidu.com/s.gif";t?(n+="?r="+encodeURIComponent(document.referrer),r&&(n+="&l="+r)):r&&(n+="?l="+r);var i=new Image;i.src=n}}(window);})();
  </script>















  

  

  

<script>
  function loadCount() {
    var d = document, s = d.createElement('script');
    s.src = 'https://zasdfgbnm-github-io.disqus.com/count.js';
    s.id = 'dsq-count-scr';
    (d.head || d.body).appendChild(s);
  }
  // defer loading until the whole page loading is completed
  window.addEventListener('load', loadCount, false);
</script>
<script>
  var disqus_config = function() {
    this.page.url = "https://zasdfgbnm.github.io/2018/06/11/%E4%BB%8E%E5%A4%B4%E5%BC%80%E5%A7%8B%E9%98%85%E8%AF%BBPyTorch%E4%BB%A3%E7%A0%81%20--%20Operators%E7%AF%87/";
    this.page.identifier = "2018/06/11/从头开始阅读PyTorch代码 -- Operators篇/";
    this.page.title = "从头开始阅读PyTorch代码 -- Operators篇";
    };
  NexT.utils.loadComments(document.querySelector('#disqus_thread'), () => {
    if (window.DISQUS) {
      DISQUS.reset({
        reload: true,
        config: disqus_config
      });
    } else {
      var d = document, s = d.createElement('script');
      s.src = 'https://zasdfgbnm-github-io.disqus.com/embed.js';
      s.setAttribute('data-timestamp', '' + +new Date());
      (d.head || d.body).appendChild(s);
    }
  });
</script>

</body>
</html>
